/**
 *  Copyright(C) 2012-2013 Suntec(Shanghai) Software Co., Ltd.
 *  All Right Reserved.
 */
package com.cugbmao.luence;

import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.UnknownHostException;

/**
 * Descriptions
 * 
 * @version 2013年5月21日
 * @author Suntec
 * @since JDK1.6
 * 
 */
public class WebParserFilter {
	private static String src_file_path = "D:\\Lucene\\html\\htmlsrc.html";
	private static String dst_file_path = "D:\\Lucene\\txt\\puresrc.txt";

	public static void main( String[] args ) throws IOException {
		try {
			parserFilter();
		}
		catch ( IOException e ) {
			System.err.println( "下载失败，请检查输入地址是否正确。" );
			System.exit( 1 );
		}
	}

	private static void parserFilter() throws IOException {
		try {
			int j = 0;
			boolean flag = true;

			boolean bContent = true;
			StringBuffer sBuffer = new StringBuffer( 8096 * 5 );
			char[] cBuffer = new char[8096 * 5];
			char[] dstBuffer = new char[8096 * 5];
			int nCount = 0;

			File srcFile = new File( src_file_path );
			FileReader fReader = new FileReader( srcFile );

			File dstFile = new File( dst_file_path );
			FileWriter fWriter = new FileWriter( dstFile );

			nCount = fReader.read( cBuffer );

			for ( int i = 0; i < nCount; i++ ) {
				if ( bContent == false ) {
					if ( cBuffer[i] == '>' ) {
						bContent = true;
					}
					else {
						continue;
					}
				}
				else {
					if ( cBuffer[i] == '<' ) {
						bContent = false;
						continue;
					}
					else if ( cBuffer[i] == '\n' || cBuffer[i] == ' ' || cBuffer[i] == '　' || cBuffer[i] == '	' ) {
						continue;
					}
					else if ( cBuffer[i] == '&' && cBuffer[i + 1] == 'n' && cBuffer[i + 2] == 'b'
							&& cBuffer[i + 3] == 's' && cBuffer[i + 4] == 'p' && cBuffer[i + 5] == ';' ) {
						i = i + 5;
						continue;
					}

					dstBuffer[j++] = cBuffer[i];
				}
			}

			for ( int i = 0; i < j; i++ ) {
				if ( dstBuffer[i] <= 'Z' && dstBuffer[i] >= 'A' || dstBuffer[i] <= 'z' && dstBuffer[i] >= 'a'
						|| dstBuffer[i] <= '9' && dstBuffer[i] >= '0' ) {
					if ( flag == false ) {
						sBuffer.append( ' ' );
					}
					sBuffer.append( dstBuffer[i] );
					flag = true;
				}
				else {
					// 过滤标点符号
					if ( dstBuffer[i] == '、' || dstBuffer[i] == '|' || dstBuffer[i] == '”' || dstBuffer[i] == '：'
							|| dstBuffer[i] == ';' || dstBuffer[i] == '.' ) {
						sBuffer.append( ' ' );
						continue;
					}
					if ( flag = true ) {
						sBuffer.append( ' ' );
					}
					// 中文字符用空格隔开
					sBuffer.append( dstBuffer[i] );
					sBuffer.append( ' ' );
					flag = false;
				}
			}

			String str = sBuffer.toString();
			System.out.println( str );
			fWriter.write( str );

			fReader.close();
			fWriter.close();
		}
		catch ( UnknownHostException e ) {
			System.err.println( "无法访问指定主机。" );
			System.exit( 1 );
		}
		catch ( IOException e ) {
			throw e;
		}
	}
}
